#!/usr/bin/env python
################################################################################
# Script for testing the prover.
#
# Author: Maxime Arthaud
#
# Contact: ikos@lists.nasa.gov
#
# Notices:
#
# Copyright (c) 2011-2019 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Disclaimers:
#
# No Warranty: THE SUBJECT SOFTWARE IS PROVIDED "AS IS" WITHOUT ANY WARRANTY OF
# ANY KIND, EITHER EXPRESSED, IMPLIED, OR STATUTORY, INCLUDING, BUT NOT LIMITED
# TO, ANY WARRANTY THAT THE SUBJECT SOFTWARE WILL CONFORM TO SPECIFICATIONS,
# ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
# OR FREEDOM FROM INFRINGEMENT, ANY WARRANTY THAT THE SUBJECT SOFTWARE WILL BE
# ERROR FREE, OR ANY WARRANTY THAT DOCUMENTATION, IF PROVIDED, WILL CONFORM TO
# THE SUBJECT SOFTWARE. THIS AGREEMENT DOES NOT, IN ANY MANNER, CONSTITUTE AN
# ENDORSEMENT BY GOVERNMENT AGENCY OR ANY PRIOR RECIPIENT OF ANY RESULTS,
# RESULTING DESIGNS, HARDWARE, SOFTWARE PRODUCTS OR ANY OTHER APPLICATIONS
# RESULTING FROM USE OF THE SUBJECT SOFTWARE.  FURTHER, GOVERNMENT AGENCY
# DISCLAIMS ALL WARRANTIES AND LIABILITIES REGARDING THIRD-PARTY SOFTWARE,
# IF PRESENT IN THE ORIGINAL SOFTWARE, AND DISTRIBUTES IT "AS IS."
#
# Waiver and Indemnity:  RECIPIENT AGREES TO WAIVE ANY AND ALL CLAIMS AGAINST
# THE UNITED STATES GOVERNMENT, ITS CONTRACTORS AND SUBCONTRACTORS, AS WELL
# AS ANY PRIOR RECIPIENT.  IF RECIPIENT'S USE OF THE SUBJECT SOFTWARE RESULTS
# IN ANY LIABILITIES, DEMANDS, DAMAGES, EXPENSES OR LOSSES ARISING FROM SUCH
# USE, INCLUDING ANY DAMAGES FROM PRODUCTS BASED ON, OR RESULTING FROM,
# RECIPIENT'S USE OF THE SUBJECT SOFTWARE, RECIPIENT SHALL INDEMNIFY AND HOLD
# HARMLESS THE UNITED STATES GOVERNMENT, ITS CONTRACTORS AND SUBCONTRACTORS,
# AS WELL AS ANY PRIOR RECIPIENT, TO THE EXTENT PERMITTED BY LAW.
# RECIPIENT'S SOLE REMEDY FOR ANY SUCH MATTER SHALL BE THE IMMEDIATE,
# UNILATERAL TERMINATION OF THIS AGREEMENT.
#
################################################################################
import os.path
import sys
current_dir = os.path.dirname(os.path.abspath(__file__))
parent_dir = os.path.dirname(current_dir)
sys.path.insert(0, parent_dir)
sys.dont_write_bytecode = True
from libruntest import TestManager, Test, parse_args

if __name__ == '__main__':
    parse_args(description='Regression tests for the prover')

    t = TestManager(root=current_dir)
    t.add(Test('01.c', '01.c', 'prover', 'safe'))
    t.add(Test('02.c', '02.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('03.c', '03.c', 'prover', 'safe'))
    t.add(Test('04.c', '04.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('05.c', '05.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('06.c', '06.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('07.c', '07.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('08.c', '08.c (interval)', 'prover', 'safe', expected='unsafe', domain='interval'))
    t.add(Test('08.c', '08.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('09.c', '09.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('10.c', '10.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('11.c', '11.c (interval)', 'prover', 'safe', expected='unsafe',
               domain='interval'))
    t.add(Test('11.c', '11.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('12.c', '12.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('13.c', '13.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('14.c', '14.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('15.c', '15.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('16.c', '16.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('17.c', '17.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('18.c', '18.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('19.c', '19.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('20.c', '20.c', 'prover', 'safe', expected='unsafe',
               line_checks=[(27, 'ok', 'warning'), (29, 'ok'), (30, 'warning')]))
    t.add(Test('21.c', '21.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('22.c', '22.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('23.c', '23.c', 'prover', 'safe'))
    t.add(Test('24.c', '24.c (interval)', 'prover', 'safe', expected='unsafe', domain='interval'))
    t.add(Test('24.c', '24.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('24.c', '24.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('25.c', '25.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('25.c', '25.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('26.c', '26.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('27.c', '27.c', 'prover', 'safe'))
    t.add(Test('28.c', '28.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('28.c', '28.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('29.c', '29.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('30.c', '30.c', 'prover', 'safe'))
    t.add(Test('31.c', '31.c', 'prover', 'safe', expected='unsafe',
               line_checks=[(19, 'ok'), (26, 'ok', 'warning')]))
    t.add(Test('32.c', '32.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('33.c', '33.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('34.c', '34.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('35.c', '35.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('36.c', '36.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('37.c', '37.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('38.c', '38.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('39.c', '39.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('39.c', '39.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('40.c', '40.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('41.c', '41.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('42.c', '42.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('43.c', '43.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('44.c', '44.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('45.c', '45.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('46.c', '46.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('47.c', '47.c', 'prover', 'safe', opt_level='aggressive'))
    t.add(Test('48-volatile-safe.c', '48-volatile-safe.c', 'prover', 'safe'))
    t.add(Test('48-volatile-unsafe.c', '48-volatile-unsafe.c', 'prover', 'unsafe'))
    t.add(Test('asian06-ex2.c', 'asian06-ex2.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('asian06-ex2.c', 'asian06-ex2.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('asian06-ex2.c', 'asian06-ex2.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('astree-1.c', 'astree-1.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('astree-1.c', 'astree-1.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('astree-1.c', 'astree-1.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('astree-2a.c', 'astree-2a.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('astree-2a.c', 'astree-2a.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('astree-2a.c', 'astree-2a.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('astree-2b.c', 'astree-2b.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('astree-2b.c', 'astree-2b.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('astree-2c.c', 'astree-2c.c', 'prover', 'safe'))
    t.add(Test('loop-1.c', 'loop-1.c', 'prover', 'safe'))
    t.add(Test('loop-2.c', 'loop-2.c', 'prover', 'safe'))
    t.add(Test('loop-3.c', 'loop-3.c', 'prover', 'safe'))
    t.add(Test('loop-4.c', 'loop-4.c', 'prover', 'safe'))
    t.add(Test('loop-9.c', 'loop-9.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('loop-9.c', 'loop-9.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('loop-10.c', 'loop-10.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('loop-10.c', 'loop-10.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('loop-10.c', 'loop-10.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('test-1.c', 'test-1.c', 'prover', 'safe'))
    t.add(Test('test-2.c', 'test-2.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-3.c', 'test-3.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-3.c', 'test-3.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('test-4.cpp', 'test-4.cpp', 'prover', 'safe', expected='unsafe',
               line_checks=[(22, 'ok'),
                            (37, 'ok'),
                            (38, 'ok'),
                            (39, 'ok'),
                            (41, 'ok'),
                            (42, 'ok', 'warning')]))
    t.add(Test('test-5.cpp', 'test-5.cpp', 'prover', 'safe'))
    t.add(Test('test-6.cpp', 'test-6.cpp', 'prover', 'safe'))
    t.add(Test('test-7.cpp', 'test-7.cpp', 'prover', 'safe'))
    t.add(Test('test-8.cpp', 'test-8.cpp', 'prover', 'safe', opt_level='none',
               line_checks=[(23, 'unreachable')]))
    t.add(Test('test-9.c', 'test-9.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-10.c', 'test-10.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-11.c', 'test-11.c', 'prover', 'safe', expected='unsafe',
               line_checks=[(21, 'ok', 'warning'), (22, 'ok', 'warning')]))
    t.add(Test('test-12.c', 'test-12.c', 'prover', 'safe',
               line_checks=[(5, 'unreachable'), (7, 'ok')]))
    t.add(Test('test-13.c', 'test-13.c', 'prover', 'safe'))
    t.add(Test('test-14.c', 'test-14.c', 'prover', 'safe'))
    t.add(Test('test-15.cpp', 'test-15.cpp', 'prover', 'safe'))
    t.add(Test('test-16.c', 'test-16.c', 'prover', 'safe'))
    t.add(Test('test-17.c', 'test-17.c', 'prover', 'safe',
               line_checks=[(19, 'ok'), (21, 'ok')]))
    t.add(Test('test-18.c', 'test-18.c (interval)', 'prover', 'safe'))
    t.add(Test('test-18.c', 'test-18.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('test-19.c', 'test-19.c (interval)', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-19.c', 'test-19.c (dbm)', 'prover', 'safe', domain='dbm'))
    t.add(Test('test-19.c', 'test-19.c (gauge-interval-congruence)', 'prover', 'safe',
               domain='gauge-interval-congruence'))
    t.add(Test('test-20.c', 'test-20.c', 'prover', 'safe'))
    t.add(Test('test-21-exceptions.cpp', 'test-21-exceptions.cpp', 'prover', 'safe'))
    t.add(Test('test-22-exceptions.cpp', 'test-22-exceptions.cpp', 'prover', 'safe'))
    t.add(Test('test-23.c', 'test-23.c', 'prover', 'safe'))
    t.add(Test('test-24.c', 'test-24.c', 'prover', 'safe'))
    t.add(Test('test-25.c', 'test-25.c', 'prover', 'unsafe'))
    t.add(Test('test-26.c', 'test-26.c', 'prover', 'unsafe',
               line_checks=[(17, 'ok'),
                            (18, 'ok'),
                            (19, 'ok'),
                            (22, 'warning'),
                            (23, 'warning'),
                            (24, 'ok'),
                            (27, 'warning'),
                            (28, 'warning'),
                            (29, 'ok', 'warning')]))
    t.add(Test('test-27.c', 'test-27.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-27.c', 'test-27.c (partitioning=return)', 'prover', 'safe',
               options=['-add-partitioning-variables',
                        '-enable-partitioning-domain']))
    t.add(Test('test-28.c', 'test-28.c', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-28.c', 'test-28.c (partitioning=return)', 'prover', 'safe',
               options=['-add-partitioning-variables',
                        '-enable-partitioning-domain']))
    t.add(Test('test-29.cpp', 'test-29.cpp', 'prover', 'safe', expected='unsafe'))
    t.add(Test('test-29.cpp', 'test-29.cpp (partitioning=return)', 'prover', 'safe',
               options=['-add-partitioning-variables',
                        '-enable-partitioning-domain']))
    t.run()
